{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "preprocessing.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "metadata": {
        "id": "7grKnL7Wn1de",
        "colab_type": "code",
        "outputId": "6724d878-8721-479e-8116-fe433a776aa3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 530
        }
      },
      "cell_type": "code",
      "source": [
        "!pip install tensorflow-gpu==2.0.0-alpha0"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting tensorflow-gpu==2.0.0-alpha0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/1a/66/32cffad095253219d53f6b6c2a436637bbe45ac4e7be0244557210dc3918/tensorflow_gpu-2.0.0a0-cp36-cp36m-manylinux1_x86_64.whl (332.1MB)\n",
            "\u001b[K    100% |████████████████████████████████| 332.1MB 55kB/s \n",
            "\u001b[?25hRequirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.7.1)\n",
            "Collecting google-pasta>=0.1.2 (from tensorflow-gpu==2.0.0-alpha0)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/64/bb/f1bbc131d6294baa6085a222d29abadd012696b73dcbf8cf1bf56b9f082a/google_pasta-0.1.5-py3-none-any.whl (51kB)\n",
            "\u001b[K    100% |████████████████████████████████| 61kB 21.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.14.6)\n",
            "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.15.0)\n",
            "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (3.7.1)\n",
            "Collecting tf-estimator-nightly<1.14.0.dev2019030116,>=1.14.0.dev2019030115 (from tensorflow-gpu==2.0.0-alpha0)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/13/82/f16063b4eed210dc2ab057930ac1da4fbe1e91b7b051a6c8370b401e6ae7/tf_estimator_nightly-1.14.0.dev2019030115-py2.py3-none-any.whl (411kB)\n",
            "\u001b[K    100% |████████████████████████████████| 419kB 10.3MB/s \n",
            "\u001b[?25hRequirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.1.0)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.33.1)\n",
            "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.2.2)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.0.9)\n",
            "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (0.7.1)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.11.0)\n",
            "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow-gpu==2.0.0-alpha0) (1.0.7)\n",
            "Collecting tb-nightly<1.14.0a20190302,>=1.14.0a20190301 (from tensorflow-gpu==2.0.0-alpha0)\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a9/51/aa1d756644bf4624c03844115e4ac4058eff77acd786b26315f051a4b195/tb_nightly-1.14.0a20190301-py3-none-any.whl (3.0MB)\n",
            "\u001b[K    100% |████████████████████████████████| 3.0MB 7.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow-gpu==2.0.0-alpha0) (40.9.0)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow-gpu==2.0.0-alpha0) (2.8.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190302,>=1.14.0a20190301->tensorflow-gpu==2.0.0-alpha0) (0.15.2)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tb-nightly<1.14.0a20190302,>=1.14.0a20190301->tensorflow-gpu==2.0.0-alpha0) (3.1)\n",
            "Installing collected packages: google-pasta, tf-estimator-nightly, tb-nightly, tensorflow-gpu\n",
            "Successfully installed google-pasta-0.1.5 tb-nightly-1.14.0a20190301 tensorflow-gpu-2.0.0a0 tf-estimator-nightly-1.14.0.dev2019030115\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "colab_type": "code",
        "outputId": "a8109dde-ea3c-40cb-83c4-379668053321",
        "id": "nScSc9GFKG11",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "cell_type": "code",
      "source": [
        "import os\n",
        "import tensorflow as tf\n",
        "from google.colab import drive\n",
        "import json\n",
        "drive.mount('gdrive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at gdrive; to attempt to forcibly remount, call drive.mount(\"gdrive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "74cR2LXFI09h",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "6f3db952-e70d-422a-9778-3fa86423552b"
      },
      "cell_type": "code",
      "source": [
        "# only need to run once -- stores data repo on google drive\n",
        "!git clone \"https://github.com/LasseRegin/medical-question-answer-data\" \"gdrive/My Drive/Colab Notebooks/medical-question-answer-data\""
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "fatal: destination path 'gdrive/My Drive/Colab Notebooks/medical-question-answer-data' already exists and is not an empty directory.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "mg1AOpKiluEq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "outputId": "499119c4-80da-440c-f8b7-4b0a78b60d14"
      },
      "cell_type": "code",
      "source": [
        "dataset_name = 'ehealthforumQAs'\n",
        "\n",
        "with open('gdrive/My Drive/Colab Notebooks/medical-question-answer-data/%s.json'%dataset_name) as file:\n",
        "  ds = json.load(file)\n",
        "  file.close()\n",
        "print('dataset length: ', len(ds))\n",
        "ds[0]"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dataset length:  171\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'answer': 'stopping smoking is about will power and being steadfast. you can stop safely by having bupropion or nicotine patch cover initially in consult with a doctor. contact an addiction clinic near you. wishing you best of health thanks',\n",
              " 'question': 'how do i stop smoking now',\n",
              " 'tags': ['addiction', 'stop smoking'],\n",
              " 'url': 'http://ehealthforum.com/health/stop-smoking-question-t462882.html'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "metadata": {
        "id": "2MyI2TA3yA0w",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 496
        },
        "outputId": "8f10f091-66f6-4886-be56-d11137ba0258"
      },
      "cell_type": "code",
      "source": [
        "examples = []\n",
        "for sample in ds:\n",
        "  question, answer, tags = sample['question'], sample['answer'], sample['tags']\n",
        "  example = tf.train.Example()\n",
        "  example.features.feature['question'].bytes_list.value.append(str.encode(question))\n",
        "  example.features.feature['answer'].bytes_list.value.append(str.encode(answer))\n",
        "  example.features.feature['tags'].bytes_list.value.extend([str.encode(tag) for tag in tags])\n",
        "  examples.append(example.SerializeToString())\n",
        "tf.train.Example.FromString(examples[0])"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "features {\n",
              "  feature {\n",
              "    key: \"answer\"\n",
              "    value {\n",
              "      bytes_list {\n",
              "        value: \"stopping smoking is about will power and being steadfast. you can stop safely by having bupropion or nicotine patch cover initially in consult with a doctor. contact an addiction clinic near you. wishing you best of health thanks\"\n",
              "      }\n",
              "    }\n",
              "  }\n",
              "  feature {\n",
              "    key: \"question\"\n",
              "    value {\n",
              "      bytes_list {\n",
              "        value: \"how do i stop smoking now\"\n",
              "      }\n",
              "    }\n",
              "  }\n",
              "  feature {\n",
              "    key: \"tags\"\n",
              "    value {\n",
              "      bytes_list {\n",
              "        value: \"addiction\"\n",
              "        value: \"stop smoking\"\n",
              "      }\n",
              "    }\n",
              "  }\n",
              "}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "metadata": {
        "id": "KbODTyf4yvBx",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "tfr = tf.data.Dataset.from_tensor_slices(examples)\n",
        "writer = tf.data.experimental.TFRecordWriter('gdrive/My Drive/Colab Notebooks/medical-question-answer-data/%s.tfrecords'%dataset_name)\n",
        "writer.write(tfr)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "UFpeCEkPGgko",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}